package com.sirui.spider.processor;

import cn.hutool.core.date.DatePattern;
import cn.hutool.core.date.DateUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.sirui.spider.kuai.entity.Kuai;
import com.sirui.spider.kuai.repository.KuaiRepository;
import com.sirui.spider.kuai.service.KuaiService;
import lombok.extern.apachecommons.CommonsLog;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@CommonsLog
@Component
public class KuaiKeJiProcessor implements PageProcessor {
    //static List<ResultItems> list = new ArrayList<ResultItems>();
    static Map<String, String> map = new HashMap<String, String>();
    @Autowired
    KuaiRepository kuaiRepository;
    @Autowired
    KuaiService kuaiService;

    public static void main(String[] args) {
        Spider.create(new KuaiKeJiProcessor()).thread(5)
                .addPipeline(new FilePipeline("d:\\logs\\webmagic\\"))
                //.setDownloader(new SeleniumDownloader("D:\\tools\\webdriver\\chromedriver_win32\\chromedriver.exe"))
                .addUrl("https://www.mydrivers.com/")
                .run();
        //.runAsync();

        log.info(String.format("A@@A r=[%s],[%s]", "over", map.keySet().size()));

    }

    public void run() {
        Spider.create(new KuaiKeJiProcessor()).thread(5)
                .addPipeline(new FilePipeline("d:\\logs\\webmagic\\"))
                //.setDownloader(new SeleniumDownloader("D:\\tools\\webdriver\\chromedriver_win32\\chromedriver.exe"))
                .addUrl("https://www.mydrivers.com/")
                .run();
        //.runAsync();

        map.keySet().stream().forEach(key -> {
            String content = map.get(key);
            Kuai kuai = kuaiRepository.findByPageUrl(key);
            if (kuai == null) {
                saveKuai(key, content);
            }
        });
    }

    public void saveKuai(String key, String content) {
        if (key.equals("https://www.mydrivers.com/")) {
            return;
        }
        Kuai kuai = new Kuai();
        kuai.setPageUrl(key);
        kuai.setContent(content);
        kuai.setCreateDate(new Date());
        JSONObject jsonObject = JSONUtil.parseObj(content);
        String title = (String) jsonObject.get("title");
        String datetime = (String) jsonObject.get("datetime");
        List<String> imgHrefList = (List<String>) jsonObject.get("imgHrefList");
        kuai.setImgCount(imgHrefList.size());
        if (StringUtils.isNotBlank(datetime)) {
            Date publishDate = DateUtil.parse(datetime, DatePattern.NORM_DATETIME_PATTERN);
            kuai.setPublishDate(publishDate);
        }
        kuai.setTitle(title);
        kuaiRepository.save(kuai);
        //if (imgHrefList.size() > 0) {
        //    log.info(String.format("A@@A [%s],[%s]", kuai.getId(), imgHrefList.size()));
        //}
    }

    @Override
    public void process(Page page) {
        //page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());

        Selectable url = page.getUrl();
        Selectable links = page.getHtml().links();

        Html html = page.getHtml();

        page.putField("url", url);
        String title = page.getHtml().xpath("//div[@id='main_contents']/div[@class='news_n']/div[@class='news_bt']/text()").toString();
        page.putField("title", title);
        String datetime = page.getHtml().xpath("//div[@id='main_contents']/div[@class='news_n']/div[@class='news_bt1']/div[@class='news_bt1_left']/text()").toString();
        if (StringUtils.isNotBlank(datetime)) {
            datetime = datetime.trim().substring(0, 19);
        }
        page.putField("datetime", datetime);

        List<String> imgHrefList = page.getHtml().xpath("//div[@id='main_contents']/div[@class='news_n']/div[@class='news_info']/p/a/@href").all();
        clearImgHrefList(imgHrefList);
        JSONObject jsonObject = JSONUtil.createObj();
        jsonObject.put("title", title);
        jsonObject.put("datetime", datetime);
        jsonObject.put("imgHrefList", imgHrefList);

        map.put(page.getUrl().get(), jsonObject.toString());

        //log.info(String.format("A@@A [%s],[%s],[%s]", page.getResultItems().get("title"), page.getResultItems().get("url"), page.getResultItems().get("datetime")));
        List<String> urlList = page.getHtml().links().regex("https://news\\.mydrivers\\.com/1/922/922.*").all();
        HashSet<String> hashSet = new HashSet<>();
        hashSet.addAll(urlList);
        page.addTargetRequests(new ArrayList<>(hashSet));

        //log.info(String.format("A@@A [%s] ,urlList=[%s],hashSet=[%s]", page.getUrl(), urlList.size(), hashSet.size()));

    }

    @Override
    public Site getSite() {
        return Site.me()
                .setDomain("www.mydrivers.com")
                .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31")
                .setCharset("UTF-8")
                .setSleepTime(500);
    }

    public void clearImgHrefList(List<String> imgHrefList) {
        Iterator<String> iterator = imgHrefList.iterator();
        while (iterator.hasNext()) {
            String content = iterator.next();
            String pattern = "\\.jpg|\\.jpeg|\\.png|\\.gif";
            Pattern r = Pattern.compile(pattern);
            Matcher m = r.matcher(content);
            if (!m.find()) {
                log.info(String.format("A@@A [%s]", content));
                iterator.remove();
            }
        }
    }
}
